import os
import sys
import requests
import json
import re
from lxml import etree

cwd = os.getcwd()
sys.path.append(cwd)
print(cwd)

from pub_func import DownloadFile, file_write


base_url = 'https://www.dreamstime.com/photos-images/{}.html'

headers = {
    'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36',
}

params = {
    # 'srh_field':'frost',
    'pg':'1',
}

if __name__ == '__main__':
    cloud_type = ['Flocculent cumulus','altocumulus translucidus','altocumulus lenticularis','altocumulus cumulogenitus','altocumulus opacus',
                'altocumulus castellanus','altostratus translucidus','altostratus opacus','cs nebu','cirrostratus fibratus',
                'cirrus fibratus','cirrus uncinus','cirrocumulus','cirrostratus nebulosus','cirrostratus fibratus',
                'nimbostratus','fracto nimbus','fractocumulus','cumulus congestus','cumulus humilis',
                'cumulonimbus capillatus','cumulonimbus calvus','fracto stratus','stratus','stratocumulus translucidus',
                'stratocumulus lenticularis','stratocumulus cumulogenitus','stratocumulus opacus','stratocumulus castellanus']
    for i in range(1, 15):
        params['pg'] = i
        response = requests.get(url=base_url.format('rainfall'), headers=headers, params=params)
        response.encoding = 'utf8'
        # print(response.text)
        html = etree.HTML(response.text)
        list_img_url = html.xpath('//*[@id="ilc0"]/a/img/@src')
        # print(list_img_url, len(list_img_url))
        print(i)
        path = os.path.join('/home/ubuntu/workspace/img' ,'dreamstime_7.csv')
        for url_img in list_img_url[:-1]:
            content = url_img + ',\n'
            file_write(path, content, 'a')
